package com.test;

import lombok.extern.slf4j.Slf4j;
import org.jsoup.Connection;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

/**
 * 沧州科技技术局
 * @author Administrator
 *
 */
@Slf4j
public class Handan  {
	
	
	/**
	 *政策法规
	 */
	public  void getArticleList(Integer uid,Integer page,String type){
		//整个html内容
		Document doc;
		int errcount = 0; //重复次数
		try {
			//http://zhannei.baidu.com/cse/site?q=%E9%82%AF%E9%83%B8%E8%AD%A6%E5%AF%9F&p=2&cc=hebei.com.cn
			Connection conn = Jsoup.connect("http://hdskjj.hd.gov.cn/web/xxgklist.aspx?newstag=hdszcfg").timeout(5000);
			//Connection conn = Jsoup.connect("http://zhannei.baidu.com/cse/site?q=" +URLEncoder.encode(keyword, "UTF-8") + "&p="+page+"&cc=hebei.com.cn").timeout(5000);
			conn.header("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9");
			conn.header("Accept-Encoding", "gzip, deflate, sdch");
			conn.header("Accept-Language", "zh-CN,zh;q=0.8");
			conn.header("User-Agent", "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/79.0.3945.88 Safari/537.36");
			conn.cookie("_gscu_318669530", "8400334245gjoh10");
			conn.cookie("ASP.NET_SessionId", "t2pph0rkz1om3pt3hcagg0vi");
			conn.cookie("_gscu_1929751046", "88901438snjjq011");
			doc = conn.get();
			String name = doc.getElementsByTag("title").text();
			log.info("********************************************第"+page+"页("+name+")********************************************");
					
			

			
			
			
			Elements tablelist = doc.select(".listcontent li");
	
			if(!tablelist.isEmpty()) {
				
				for (Element info : tablelist) {
					//System.out.println(info.html());
					
					String title = info.select("a").first().text();
					String url = info.select("a").first().attr("href");
					String time = info.select("span").text();
					
					
//					if(StringUtils.isNotEmpty(url)) {		//爬虫过滤重复url
//						if(redisService.includeByBloomFilter(bloomFilterHelper, "www.hebei.com.cn"+type, url)){  //url已存在
//							errcount++;
//						}else {
//							redisService.addByBloomFilter(bloomFilterHelper, "www.hebei.com.cn"+type, url);
//							if(WebCrawlerCacheUtils.getTotalCount(url, type) > 0) {
//								errcount++;
//								break;
//							}else {
//								Date date = DateUtils.stringToDate(time, "yyyy-MM-dd");
//								WebCrawlerCacheUtils.addArticle(title, url, date, type, "长城网");
//							}
//						}
//					}
					
					log.info(title);
					log.info(url);
					log.info(time);
					
					log.info("----------------------------------------重复次数"+errcount+"--------------------------------------------------");
				}
			}

			//查询分页列表
			page++;
			if(!tablelist.isEmpty() && errcount < 8) {
				//get_list(keyword, page,type);
			}
		} catch (Exception e) {
			// TODO Auto-generated catch block
			e.printStackTrace();
		} // 设置连接超时时间 
	}
	
	
	
	public static void main(String[] args) {
		Handan chengde = new Handan();
		chengde.getArticleList(2781, 1, "政策文件");
		chengde.getArticleList(2782, 1, "政策解读");
	}
}
